-
Notifications
You must be signed in to change notification settings - Fork 129
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Chmod 777 main #135
Open
Recepxx34
wants to merge
2
commits into
gateio:master
Choose a base branch
from
Recepxx34:patch-2
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Chmod 777 main #135
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
* * By using the software, you agree to * the Visual Studio Code Server License Terms (https://aka.ms/vscode-server-license) and * the Microsoft Privacy Statement (https://privacy.microsoft.com/en-US/privacystatement).
import ( "encoding/hex" "encoding/json" "fmt" "gate-zkmerkle-proof/config" "gate-zkmerkle-proof/global" "gate-zkmerkle-proof/utils" "github.com/consensys/gnark-crypto/ecc/bn254/fr/poseidon" zk_smt "github.com/gatechain/gate-zk-smt" "gorm.io/driver/mysql" "gorm.io/gorm" "gorm.io/gorm/logger" "io/ioutil" "log" "os" "time" ) func HandleUserData(userProofConfig *config.Config) []utils.AccountInfo { startTime := time.Now().UnixMilli() accounts, _, err := utils.ReadUserAssets(userProofConfig.UserDataFile) if err != nil { panic(err.Error()) } endTime := time.Now().UnixMilli() fmt.Println("handle user data cost ", endTime-startTime, " ms") return accounts } type AccountLeave struct { hash []byte index uint32 } func ComputeAccountRootHash(userProofConfig *config.Config) { accountTree, err := utils.NewAccountTree("memory", "") if err != nil { panic(err.Error()) } accounts, _, err := utils.ReadUserAssets(userProofConfig.UserDataFile) if err != nil { panic(err.Error()) } startTime := time.Now().UnixMilli() totalOpsNumber := len(accounts) fmt.Println("total ops number is ", totalOpsNumber) chs := make(chan AccountLeave, 1000) workers := 32 results := make(chan bool, workers) averageAccounts := (totalOpsNumber + workers - 1) / workers actualWorkers := 0 for i := 0; i < workers; i++ { srcAccountIndex := i * averageAccounts destAccountIndex := (i + 1) * averageAccounts if destAccountIndex > totalOpsNumber { destAccountIndex = totalOpsNumber } go CalculateAccountHash(accounts[srcAccountIndex:destAccountIndex], chs, results) if destAccountIndex == totalOpsNumber { actualWorkers = i + 1 break } } fmt.Println("actual workers is ", actualWorkers) quit := make(chan bool, 1) go CalculateAccountTreeRoot(chs, &accountTree, quit) for i := 0; i < actualWorkers; i++ { <-results } close(chs) <-quit endTime := time.Now().UnixMilli() fmt.Println("user account tree generation cost ", endTime-startTime, " ms") fmt.Printf("account tree root %x\n", accountTree.Root()) } func CalculateAccountHash(accounts []utils.AccountInfo, chs chan<- AccountLeave, res chan<- bool) { poseidonHasher := poseidon.NewPoseidon() for i := 0; i < len(accounts); i++ { chs <- AccountLeave{ hash: utils.AccountInfoToHash(&accounts[i], &poseidonHasher), index: accounts[i].AccountIndex, } } res <- true } func CalculateAccountTreeRoot(accountLeaves <-chan AccountLeave, accountTree *zk_smt.SparseMerkleTree, quit chan<- bool) { num := 0 for accountLeaf := range accountLeaves { (*accountTree).Set(uint64(accountLeaf.index), accountLeaf.hash) num++ if num%100000 == 0 { fmt.Println("for now, already set ", num, " accounts in tree") } } quit <- true } func Handler() { global.Cfg = &config.Config{} jsonFile, err := ioutil.ReadFile("./config/config.json") if err != nil { panic(fmt.Sprintf("load config err : %s", err.Error())) } err = json.Unmarshal(jsonFile, global.Cfg) if err != nil { panic(err.Error()) } userProofConfig := global.Cfg accountTree, err := utils.NewAccountTree(userProofConfig.TreeDB.Driver, userProofConfig.TreeDB.Option.Addr) accounts := HandleUserData(userProofConfig) fmt.Println("num", len(accounts)) userProofModel := OpenUserProofTable(userProofConfig) latestAccountIndex, err := userProofModel.GetLatestAccountIndex() if err != nil && err != utils.DbErrNotFound { panic(err.Error()) } if err == nil { latestAccountIndex += 1 } accountTreeRoot := hex.EncodeToString(accountTree.Root()) jobs := make(chan Job, 1000) nums := make(chan int, 1) results := make(chan *UserProof, 1000) for i := 0; i < 1; i++ { go worker(jobs, results, nums, accountTreeRoot) } quit := make(chan int, 1) for i := 0; i < 1; i++ { go WriteDB(results, userProofModel, quit, latestAccountIndex) } for i := int(latestAccountIndex); i < len(accounts); i++ { leaf, err := accountTree.Get(uint64(i), nil) if err != nil { panic(err.Error()) } proof, err := accountTree.GetProof(uint64(accounts[i].AccountIndex)) if err != nil { panic(err.Error()) } jobs <- Job{ account: &accounts[i], proof: proof, leaf: leaf, } } close(jobs) totalCounts := int(latestAccountIndex) for i := 0; i < 1; i++ { num := <-nums totalCounts += num fmt.Println("totalCounts", totalCounts) } if totalCounts != len(accounts) { fmt.Println("totalCounts actual:expected", totalCounts, len(accounts)) panic("mismatch num") } close(results) for i := 0; i < 1; i++ { <-quit } fmt.Println("userproof service run finished...") } func WriteDB(results <-chan *UserProof, userProofModel UserProofModel, quit chan<- int, latestAccountIndex uint32) { index := 0 proofs := make([]UserProof, 100) num := int(latestAccountIndex) for proof := range results { proofs[index] = *proof index += 1 if index%100 == 0 { error := userProofModel.CreateUserProofs(proofs) if error != nil { panic(error.Error()) } num += 100 if num%100000 == 0 { fmt.Println("write ", num, "proof to db") } index = 0 } } proofs = proofs[:index] if index > 0 { fmt.Println("write ", len(proofs), "proofs to db") userProofModel.CreateUserProofs(proofs) num += index } fmt.Println("total write ", num) quit <- 0 } type Job struct { account *utils.AccountInfo proof [][]byte leaf []byte } func worker(jobs <-chan Job, results chan<- *UserProof, nums chan<- int, root string) { num := 0 for job := range jobs { userProof := ConvertAccount(job.account, job.leaf, job.proof, root) results <- userProof num += 1 } nums <- num } func ConvertAccount(account *utils.AccountInfo, leafHash []byte, proof [][]byte, root string) *UserProof { var userProof UserProof var userConfig UserConfig userProof.AccountIndex = account.AccountIndex userProof.AccountId = hex.EncodeToString(account.AccountId) userProof.AccountLeafHash = hex.EncodeToString(leafHash) proofSerial, err := json.Marshal(proof) userProof.Proof = string(proofSerial) assets, err := json.Marshal(account.Assets) if err != nil { panic(err.Error()) } userProof.Assets = string(assets) userProof.TotalDebt = account.TotalDebt.String() userProof.TotalEquity = account.TotalEquity.String() userConfig.Arrangement = account.AccountIndex userConfig.UniqueIdentification = hex.EncodeToString(account.AccountId) userConfig.MerkleProofEncode = proof userConfig.TreeRootHash = root userConfig.AssetDetails = account.Assets userConfig.TotalAssetDebt = account.TotalDebt userConfig.TotalAssetEquity = account.TotalEquity configSerial, err := json.Marshal(userConfig) if err != nil { panic(err.Error()) } userProof.Config = string(configSerial) return &userProof } func OpenUserProofTable(userConfig *config.Config) UserProofModel { newLogger := logger.New( log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer logger.Config{ SlowThreshold: 60 * time.Second, // Slow SQL threshold LogLevel: logger.Silent, // Log level IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger Colorful: false, // Disable color }, ) db, err := gorm.Open(mysql.Open(userConfig.MysqlDataSource), &gorm.Config{ Logger: newLogger, }) if err != nil { panic(err.Error()) } userProofTable := NewUserProofModel(db, userConfig.DbSuffix) userProofTable.CreateUserProofTable() return userProofTable }
Fart10
approved these changes
Nov 20, 2024
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't now what is it(
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
import (
"encoding/hex"
"encoding/json"
"fmt"
"gate-zkmerkle-proof/config"
"gate-zkmerkle-proof/global"
"gate-zkmerkle-proof/utils"
"github.com/consensys/gnark-crypto/ecc/bn254/fr/poseidon"
zk_smt "github.com/gatechain/gate-zk-smt"
"gorm.io/driver/mysql"
"gorm.io/gorm"
"gorm.io/gorm/logger"
"io/ioutil"
"log"
"os"
"time"
)
func HandleUserData(userProofConfig *config.Config) []utils.AccountInfo {
startTime := time.Now().UnixMilli()
accounts, _, err := utils.ReadUserAssets(userProofConfig.UserDataFile)
if err != nil {
panic(err.Error())
}
}
type AccountLeave struct {
hash []byte
index uint32
}
func ComputeAccountRootHash(userProofConfig *config.Config) {
accountTree, err := utils.NewAccountTree("memory", "")
if err != nil {
panic(err.Error())
}
accounts, _, err := utils.ReadUserAssets(userProofConfig.UserDataFile)
if err != nil {
panic(err.Error())
}
startTime := time.Now().UnixMilli()
totalOpsNumber := len(accounts)
fmt.Println("total ops number is ", totalOpsNumber)
chs := make(chan AccountLeave, 1000)
workers := 32
results := make(chan bool, workers)
averageAccounts := (totalOpsNumber + workers - 1) / workers
actualWorkers := 0
for i := 0; i < workers; i++ {
srcAccountIndex := i * averageAccounts
destAccountIndex := (i + 1) * averageAccounts
if destAccountIndex > totalOpsNumber {
destAccountIndex = totalOpsNumber
}
go CalculateAccountHash(accounts[srcAccountIndex:destAccountIndex], chs, results)
if destAccountIndex == totalOpsNumber {
actualWorkers = i + 1
break
}
}
fmt.Println("actual workers is ", actualWorkers)
quit := make(chan bool, 1)
go CalculateAccountTreeRoot(chs, &accountTree, quit)
}
func CalculateAccountHash(accounts []utils.AccountInfo, chs chan<- AccountLeave, res chan<- bool) {
poseidonHasher := poseidon.NewPoseidon()
for i := 0; i < len(accounts); i++ {
chs <- AccountLeave{
hash: utils.AccountInfoToHash(&accounts[i], &poseidonHasher),
index: accounts[i].AccountIndex,
}
}
res <- true
}
func CalculateAccountTreeRoot(accountLeaves <-chan AccountLeave, accountTree *zk_smt.SparseMerkleTree, quit chan<- bool) {
num := 0
for accountLeaf := range accountLeaves {
(*accountTree).Set(uint64(accountLeaf.index), accountLeaf.hash)
num++
if num%100000 == 0 {
fmt.Println("for now, already set ", num, " accounts in tree")
}
}
quit <- true
}
func Handler() {
global.Cfg = &config.Config{}
jsonFile, err := ioutil.ReadFile("./config/config.json")
if err != nil {
panic(fmt.Sprintf("load config err : %s", err.Error()))
}
err = json.Unmarshal(jsonFile, global.Cfg)
if err != nil {
panic(err.Error())
}
}
func WriteDB(results <-chan *UserProof, userProofModel UserProofModel, quit chan<- int, latestAccountIndex uint32) {
index := 0
proofs := make([]UserProof, 100)
num := int(latestAccountIndex)
for proof := range results {
proofs[index] = *proof
index += 1
if index%100 == 0 {
error := userProofModel.CreateUserProofs(proofs)
if error != nil {
panic(error.Error())
}
num += 100
if num%100000 == 0 {
fmt.Println("write ", num, "proof to db")
}
index = 0
}
}
proofs = proofs[:index]
if index > 0 {
fmt.Println("write ", len(proofs), "proofs to db")
userProofModel.CreateUserProofs(proofs)
num += index
}
fmt.Println("total write ", num)
quit <- 0
}
type Job struct {
account *utils.AccountInfo
proof [][]byte
leaf []byte
}
func worker(jobs <-chan Job, results chan<- *UserProof, nums chan<- int, root string) {
num := 0
for job := range jobs {
userProof := ConvertAccount(job.account, job.leaf, job.proof, root)
results <- userProof
num += 1
}
nums <- num
}
func ConvertAccount(account *utils.AccountInfo, leafHash []byte, proof [][]byte, root string) *UserProof {
var userProof UserProof
var userConfig UserConfig
userProof.AccountIndex = account.AccountIndex
userProof.AccountId = hex.EncodeToString(account.AccountId)
userProof.AccountLeafHash = hex.EncodeToString(leafHash)
proofSerial, err := json.Marshal(proof)
userProof.Proof = string(proofSerial)
assets, err := json.Marshal(account.Assets)
if err != nil {
panic(err.Error())
}
userProof.Assets = string(assets)
userProof.TotalDebt = account.TotalDebt.String()
userProof.TotalEquity = account.TotalEquity.String()
}
func OpenUserProofTable(userConfig *config.Config) UserProofModel {
newLogger := logger.New(
log.New(os.Stdout, "\r\n", log.LstdFlags), // io writer
logger.Config{
SlowThreshold: 60 * time.Second, // Slow SQL threshold
LogLevel: logger.Silent, // Log level
IgnoreRecordNotFoundError: true, // Ignore ErrRecordNotFound error for logger
Colorful: false, // Disable color
},
)
db, err := gorm.Open(mysql.Open(userConfig.MysqlDataSource), &gorm.Config{
Logger: newLogger,
})
if err != nil {
panic(err.Error())
}
userProofTable := NewUserProofModel(db, userConfig.DbSuffix)
userProofTable.CreateUserProofTable()
return userProofTable
}